package com.bmth.alise.client;

import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang.StringUtils;

import com.bmth.alise.utils.commom.CommonUtil;
import com.bmth.alise.utils.es.ElasticsearchTemplate;
import com.bmth.alise.utils.mq.MQSender;
import com.bmth.alise.utils.rds.RedisUtils;

import net.sf.json.JSONObject;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

public class CommonPageProcessor implements PageProcessor {

	private Site site;
	private String regex;

	public CommonPageProcessor(Site site, String regex) {
		this.regex = regex;
		this.site = site;
	}

	Pattern p = Pattern.compile(".*\\.([\\w\\d]+\\.[\\w\\d]+)");

	@Override
	public void process(Page page) {
		try {
			String url = page.getUrl().get();

			List<String> allUrl = page.getHtml().links().regex("(" + regex + ")").all();
			System.out.println(allUrl.size());
			Set<String> allUrlSet = new TreeSet<>(allUrl);
			System.out.println(allUrlSet.size());
			List<String> newAllUrl = new ArrayList<>();

			for (String newUrl : allUrlSet) {
				// System.out.println(newUrl);
				try {
					newUrl = newUrl.contains("#") ? newUrl.substring(0, newUrl.indexOf("#")) : newUrl;
					if (StringUtils.isNotBlank(newUrl)) {
						// redis中是否存在。如果不纯在加入redis中，并且添加新url集合中
						boolean exists = RedisUtils.hexists("all:urls", CommonUtil.md5Password(newUrl));
						if (!exists) {
							RedisUtils.hset("all:urls", CommonUtil.md5Password(newUrl), newUrl);
							newAllUrl.add(newUrl);
						}
					}
				} catch (Exception e) {
					e.printStackTrace();
				}
			}
			System.out.println("------");

			url = url.contains("#") ? url.substring(0, url.indexOf("#")) : url;
			URL u = new URL(url);
			String host = u.getHost();
			Matcher matcher = p.matcher(host);
			String mainHost = matcher.find() ? matcher.group(1) : "";
			if (newAllUrl != null && newAllUrl.size() > 0) {
				MQSender.send(mainHost, newAllUrl);
				// MQSender.getInstance().send(mainHost, newAllUrl);
			}
			String html = page.getHtml().toString();
			String md5 = CommonUtil.md5Password(url);
			JSONObject docJson = new JSONObject();
			docJson.accumulate("url", url);
			docJson.accumulate("html", html);
			docJson.accumulate("index_date", new Date());
			ElasticsearchTemplate.addDoc(md5, docJson.toString());
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	@Override
	public Site getSite() {
		return site;
	}
}